#!/usr/bin/env python
#encoding=utf-8
import os,sys,re,MySQLdb
#from threadpool import ThreadPool
from warnings import filterwarnings
import threading
reload(sys)
sys.setdefaultencoding("utf8")
MY_CONFIG={"host":"10.9.35.226","user":"test","db":"loadtest","passwd":"test@123","charset":"utf8"}
INSERT_SQL="insert into requests (id,app,create_datetime,request_uri,request_data,transaction,version,response_time) values(NULL,'shark',unix_timestamp(%s),%s,%s,%s,%s,%s)"
COMMAND="""grep -E 'request_uri=.*state=Success' %s | awk '{print $1,substr($2,0,9),$6,$7,$8,$9,$10}'"""
PATTERN_STR="(\d{4,}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2})\srequest_uri=(.*)\srequest_data=(.*)\stransaction=(.*)\sversion=(.*)\sresponse_time=(.*)\s"

class LoadTest(object):
	def __init__(self,app_name):
		self.total_insert_rows=0
		if app_name is not None:
			self.insert_sql=INSERT_SQL.replace("shark",app_name)
		else:
			self.insert_sql=INSERT_SQL

	def __getFileList(self,fn):
		file_list=list()
		if os.path.isdir(fn):
			fn_list=os.listdir(fn)
			for i in fn_list:
				tmp_fn=os.path.join(fn,i)
				if os.path.isfile(tmp_fn) and os.path.getsize(fn)>0:
					file_list.append(tmp_fn)
		elif os.path.getsize(fn)>0:
			file_list.append(fn)
		return file_list



	def __pushlog2db(self,fn):
		conn=None
		cur=None
		filterwarnings('ignore',category=MySQLdb.Warning)
		pattern=re.compile(PATTERN_STR)
		try:
			result_tmp =re.findall(pattern,os.popen(COMMAND % fn).read())
			if result_tmp:
				conn=MySQLdb.connect(**MY_CONFIG)
				cur=conn.cursor()
				current_rows_count=cur.executemany(self.insert_sql,result_tmp)
				conn.commit()
				print("从%s,插入%d条记录%s" % (fn,current_rows_count,os.linesep))
				
		except IOError,e:
			print e
		except MySQLdb.Error,b:
			print b
		finally:
			if cur is not None:
				cur.close()
			if conn is not None:
				conn.close()

	def pushlog2db_ext(self,fn):
		file_list=self.__getFileList(fn)
		if file_list:
			for i in file_list:
				threading.Thread(target=self.__pushlog2db,args=(i,)).start()

if __name__=="__main__":

	if len(sys.argv)==3:
 		if os.path.exists(sys.argv[2]):
			ld=LoadTest(sys.argv[1])
			ld.pushlog2db_ext(sys.argv[2])
		else:
			print("%s 不存在" % sys.argv[2])

	else:
		print """
		pushlog2db <webapp_name>  <日志文件|存放日志文件的目录>
		如:
		pushlog2db shark /data/logs/
		pushlog2db ssapinew /data/www/autoshark/runtime/logs/requests.log
		"""

